{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {},
   "outputs": [],
   "source": [
    "from circleci.api import Api\n",
    "import pprint\n",
    "from collections import defaultdict\n",
    "\n",
    "token = open(\".env\").readlines()[0].split(\"=\")[1].strip()\n",
    "circleci = Api(token)\n",
    "\n",
    "# get info about your user\n",
    "#pprint.pprint(circleci.get_user_info())\n",
    "\n",
    "# get list of all of your projects\n",
    "# --> build_num --> get_build_info() -> steps -> output_url -> fetch it -> x[0][\"message\"] is newline delim string\n",
    "# --> .outcome == \"failed\" e.g.\n",
    "\n",
    "results = defaultdict(lambda: defaultdict(int))\n",
    "\n",
    "# get results 100 at a time\n",
    "\n",
    "get_how_many = 32\n",
    "per_page = 100\n",
    "pages = max(1, int(get_how_many / per_page))\n",
    "builds = []\n",
    "\n",
    "for i in range(pages):\n",
    "    for build in circleci.get_project_build_summary(\n",
    "            \"pachyderm\", \"pachyderm\", limit=min(100, get_how_many), offset=i*per_page,\n",
    "        ):\n",
    "        builds.append(build)\n",
    "        outcome = build[\"outcome\"]\n",
    "        build_num = build[\"build_num\"]\n",
    "\n",
    "        if not outcome == \"failed\" and not outcome == \"success\":\n",
    "            continue\n",
    "\n",
    "        job = build[\"workflows\"][\"job_name\"]\n",
    "        #print(f\"build {build_num} {outcome} {job}\")\n",
    "        if job.startswith(\"test-\"):\n",
    "            results[job][outcome] += 1\n",
    "\n",
    "#builds = builds[:12]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finding flakiest test suites\n",
    "\n",
    "The following produces a table of test suites ordered by flakiest suite first."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>failed</th>\n",
       "      <th>success</th>\n",
       "      <th>pass_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>test-MISC</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-PFS</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-PPS4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-PPS2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-PPS3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-PPS5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-PPS1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-ADMIN</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-EXAMPLES</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-AUTH1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-PPS6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test-AUTH2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               failed  success  pass_rate\n",
       "test-MISC         1.0      1.0        0.5\n",
       "test-PFS          1.0      1.0        0.5\n",
       "test-PPS4         1.0      1.0        0.5\n",
       "test-PPS2         0.0      2.0        1.0\n",
       "test-PPS3         0.0      1.0        1.0\n",
       "test-PPS5         0.0      2.0        1.0\n",
       "test-PPS1         0.0      2.0        1.0\n",
       "test-ADMIN        0.0      2.0        1.0\n",
       "test-EXAMPLES     0.0      2.0        1.0\n",
       "test-AUTH1        0.0      2.0        1.0\n",
       "test-PPS6         0.0      2.0        1.0\n",
       "test-AUTH2        0.0      1.0        1.0"
      ]
     },
     "execution_count": 416,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame.from_dict(results)\n",
    "\n",
    "# Transpose\n",
    "df = df.T\n",
    "\n",
    "# NaN -> 0\n",
    "df = df.fillna(0)\n",
    "\n",
    "df['pass_rate'] = df['success'] / (df['failed'] + df['success'])\n",
    "\n",
    "# sort by failed\n",
    "df = df.sort_values(by=[\"pass_rate\"])\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finding flakiest individual tests\n",
    "Now we fetch the logs for each individual test and find the flakiest individual tests."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "cache = {}\n",
    "build_info_cache = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [],
   "source": [
    "build_map = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_build(build):\n",
    "    tests = []\n",
    "    failed = set()\n",
    "    passed = set()\n",
    "    skipped = set()\n",
    "    try:\n",
    "        output_url = build[\"steps\"][5][\"actions\"][0][\"output_url\"]\n",
    "    except Exception as e:\n",
    "        print(f\"Got error: {e}, continuing...\")\n",
    "        return set(), set(), set()\n",
    "    \n",
    "    if output_url in cache:\n",
    "        lines = cache[output_url]\n",
    "    else:\n",
    "        lines = requests.get(output_url).json()\n",
    "        cache[output_url] = lines\n",
    "\n",
    "    for line in lines[0][\"message\"].split(\"\\n\"):\n",
    "        #print(line)\n",
    "        if \"RUN\" in line:\n",
    "            parts = line.split(\"RUN\")\n",
    "            if len(parts) != 2:\n",
    "                continue\n",
    "            preamble, test = parts\n",
    "            test = test.strip()\n",
    "            # ignore docker RUN lines which contain e.g. \"#10\"\n",
    "            if \"#\" in preamble or \"Step\" in preamble:\n",
    "                continue\n",
    "            if len(test) > 100:\n",
    "                # some base64 gunk\n",
    "                continue\n",
    "            if \"AcceptEnv\" in test or \"cd pachyderm\" in test \\\n",
    "                    or \"git clone\" in test or \"NING\" in test or \"_BAD_TESTS\" in test:\n",
    "                continue\n",
    "            tests.append(test)\n",
    "        if \"FAIL\" in line:\n",
    "            test = line.split(\"FAIL\")[1].replace(\":\", \"\").strip().split(\"(\")[0].strip()\n",
    "            if \"github.com\" in test or test == \"\":\n",
    "                # filter out some more noise\n",
    "                continue\n",
    "            if len(test) > 100:\n",
    "                # some base64 gunk\n",
    "                continue\n",
    "            failed.add(test)\n",
    "        if \"PASS\" in line:\n",
    "            test = line.split(\"PASS\")[1].replace(\":\", \"\").strip().split(\"(\")[0].strip()\n",
    "            if test == \"\":\n",
    "                # This happens when all the tests pass, we get a \"PASS\" on its own.\n",
    "                continue\n",
    "            if len(test) > 100:\n",
    "                # some base64 gunk\n",
    "                continue\n",
    "            if \"\\\\n\" in test:\n",
    "                continue\n",
    "            passed.add(test)\n",
    "        if \"SKIP\" in line:\n",
    "            test = line.split(\"SKIP\")[1].replace(\":\", \"\").strip().split(\"(\")[0].strip()\n",
    "            if test == \"\":\n",
    "                # This happens when all the tests pass, we get a \"PASS\" on its own.\n",
    "                continue\n",
    "            if len(test) > 100:\n",
    "                # some base64 gunk\n",
    "                continue\n",
    "            if \"\\\\n\" in test:\n",
    "                continue\n",
    "            skipped.add(test)\n",
    "\n",
    "    all_tests = set(tests)\n",
    "    for test in all_tests:\n",
    "        if test not in build_map:\n",
    "            build_map[test] = build[\"workflows\"][\"job_name\"], \\\n",
    "            f\"<a target='_blank' href='{build['build_url']}'>{build['build_num']}</a>\"\n",
    "    hung = all_tests - failed - passed - skipped\n",
    "    assert all_tests == (failed | passed | hung | skipped), \\\n",
    "        f\"all={all_tests}, failed={failed}, passed={passed}, hung={hung}, skipped={skipped}\"\n",
    "    return passed, failed, hung"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".Got error: list index out of range, continuing...\n",
      ".Got error: list index out of range, continuing...\n",
      ".Got error: list index out of range, continuing...\n",
      ".Got error: list index out of range, continuing...\n",
      ".Got error: list index out of range, continuing...\n",
      ".Got error: list index out of range, continuing...\n",
      ".Got error: list index out of range, continuing...\n",
      ".Got error: list index out of range, continuing...\n",
      ".Got error: list index out of range, continuing...\n",
      ".Got error: list index out of range, continuing...\n",
      "......................"
     ]
    }
   ],
   "source": [
    "build_results = defaultdict(lambda: defaultdict(int))\n",
    "\n",
    "for build in builds:\n",
    "    print(\".\", end=\"\")\n",
    "    if build[\"build_num\"] in build_info_cache:\n",
    "        build_info = build_info_cache[build[\"build_num\"]]\n",
    "    else:\n",
    "        build_info = circleci.get_build_info(\"pachyderm\", \"pachyderm\", build[\"build_num\"])\n",
    "        build_info_cache[build[\"build_num\"]] = build_info\n",
    "    passed, failed, hung = parse_build(build_info)\n",
    "    for b in passed:\n",
    "        build_results[b][\"pass\"] += 1\n",
    "        build_results[b][\"bucket\"] = build_map[b][0]\n",
    "    for b in failed:\n",
    "        build_results[b][\"fail\"] += 1\n",
    "        build_results[b][\"bucket\"] = build_map[b][0]\n",
    "        build_results[b][\"example\"] = build_map[b][1]\n",
    "    for b in hung:\n",
    "        build_results[b][\"hung\"] += 1\n",
    "        build_results[b][\"bucket\"] = build_map[b][0]\n",
    "        build_results[b][\"example\"] = build_map[b][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 421,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'fail'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2888\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2889\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2890\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'fail'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-421-d2968774b94e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'hang_rate'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'hung'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'pass'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'fail'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'hung'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'fail_rate'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'fail'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'pass'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'fail'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'hung'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'hang_or_fail'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'hang_rate'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'fail_rate'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2897\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2898\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2899\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2900\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2901\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2889\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2890\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2891\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2892\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2893\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtolerance\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'fail'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame.from_dict(build_results)\n",
    "\n",
    "# Transpose\n",
    "df = df.T\n",
    "\n",
    "# NaN -> 0\n",
    "df = df.fillna(0)\n",
    "\n",
    "df['hang_rate'] = df['hung'] / (df['pass'] + df['fail'] + df['hung'])\n",
    "df['fail_rate'] = df['fail'] / (df['pass'] + df['fail'] + df['hung'])\n",
    "df['hang_or_fail'] = df['hang_rate'] + df['fail_rate']\n",
    "\n",
    "hangy = df[df[\"hang_rate\"] > 0]\n",
    "faily = df[df[\"fail_rate\"] > 0]\n",
    "\n",
    "# sort by failed\n",
    "hangy = hangy.sort_values(by=[\"hang_rate\"], ascending=False)\n",
    "faily = hangy.sort_values(by=[\"fail_rate\"], ascending=False)\n",
    "\n",
    "bad = df[df[\"hang_or_fail\"] > 0]\n",
    "bad = bad.sort_values(by=[\"hang_or_fail\"], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pandas.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import HTML\n",
    "HTML(bad.to_html(escape=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
